{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "初始化Q表完成\n",
      "Q_Table,0\n",
      "episode is 0\n",
      "Random_start:46\n",
      "19\n",
      "0.999811917868\n",
      "episode is 1\n",
      "Random_start:6\n",
      "1062\n",
      "0.989541831169\n",
      "episode is 2\n",
      "Random_start:84\n",
      "1351\n",
      "0.986715041955\n",
      "episode is 3\n",
      "Random_start:72\n",
      "1513\n",
      "0.985134044541\n",
      "episode is 4\n",
      "Random_start:79\n",
      "1880\n",
      "0.981561861563\n",
      "episode is 5\n",
      "Normal_start:1\n",
      "2264\n",
      "0.977838217986\n",
      "episode is 6\n",
      "Random_start:37\n",
      "2940\n",
      "0.971317695828\n",
      "episode is 7\n",
      "Random_start:57\n",
      "3771\n",
      "0.963362246449\n",
      "episode is 8\n",
      "Normal_start:1\n",
      "4265\n",
      "0.958664250555\n",
      "episode is 9\n",
      "Normal_start:1\n",
      "4578\n",
      "0.955699573591\n",
      "episode is 10\n",
      "Random_start:33\n",
      "4698\n",
      "0.954565414734\n",
      "episode is 11\n",
      "Random_start:33\n",
      "5293\n",
      "0.948961937393\n",
      "episode is 12\n",
      "Random_start:23\n",
      "5710\n",
      "0.945054618536\n",
      "episode is 13\n",
      "Normal_start:1\n",
      "5761\n",
      "0.944577862263\n",
      "episode is 14\n",
      "Random_start:16\n",
      "6138\n",
      "0.941061136915\n",
      "episode is 15\n",
      "Normal_start:1\n",
      "6443\n",
      "0.938225726646\n",
      "episode is 16\n",
      "Random_start:7\n",
      "6763\n",
      "0.935260151771\n",
      "episode is 17\n",
      "Random_start:14\n",
      "7014\n",
      "0.932940660969\n",
      "episode is 18\n",
      "Random_start:84\n",
      "7041\n",
      "0.932691500629\n",
      "episode is 19\n",
      "Normal_start:1\n",
      "7122\n",
      "0.931944423121\n",
      "episode is 20\n",
      "Random_start:75\n",
      "7149\n",
      "0.931695531728\n",
      "episode is 21\n",
      "Random_start:12\n",
      "7244\n",
      "0.930820336756\n",
      "episode is 22\n",
      "Normal_start:1\n",
      "7372\n",
      "0.92964244074\n",
      "episode is 23\n",
      "Random_start:61\n",
      "7686\n",
      "0.926759292388\n",
      "episode is 24\n",
      "Random_start:21\n",
      "7761\n",
      "0.926071980692\n",
      "episode is 25\n",
      "Random_start:30\n",
      "7878\n",
      "0.925000803236\n",
      "episode is 26\n",
      "Normal_start:1\n",
      "8101\n",
      "0.922962624858\n",
      "episode is 27\n",
      "Normal_start:1\n",
      "8288\n",
      "0.921256980025\n",
      "episode is 28\n",
      "Random_start:16\n",
      "8536\n",
      "0.918999862697\n",
      "episode is 29\n",
      "Normal_start:1\n",
      "8944\n",
      "0.915298698765\n",
      "episode is 30\n",
      "Random_start:63\n",
      "9284\n",
      "0.912225909891\n",
      "episode is 31\n",
      "Random_start:15\n",
      "9626\n",
      "0.909145567667\n",
      "episode is 32\n",
      "Random_start:12\n",
      "9799\n",
      "0.907591390585\n",
      "episode is 33\n",
      "Normal_start:1\n",
      "10067\n",
      "0.905189066211\n",
      "episode is 34\n",
      "Random_start:82\n",
      "10091\n",
      "0.904974246615\n",
      "episode is 35\n",
      "Normal_start:1\n",
      "10192\n",
      "0.904070778954\n",
      "episode is 36\n",
      "Random_start:95\n",
      "10208\n",
      "0.903927739072\n",
      "episode is 37\n",
      "Random_start:45\n",
      "10310\n",
      "0.903016397642\n",
      "episode is 38\n",
      "Normal_start:1\n",
      "10363\n",
      "0.902543224353\n",
      "episode is 39\n",
      "Normal_start:1\n",
      "10462\n",
      "0.901660043807\n",
      "episode is 40\n",
      "Random_start:33\n",
      "10591\n",
      "0.900510543938\n",
      "episode is 41\n",
      "Random_start:29\n",
      "10650\n",
      "0.89998529768\n",
      "episode is 42\n",
      "Random_start:22\n",
      "11080\n",
      "0.896166577033\n",
      "episode is 43\n",
      "Random_start:69\n",
      "11214\n",
      "0.894979909065\n",
      "episode is 44\n",
      "Random_start:32\n",
      "11416\n",
      "0.893194053969\n",
      "episode is 45\n",
      "Normal_start:1\n",
      "11538\n",
      "0.892117214229\n",
      "episode is 46\n",
      "Random_start:10\n",
      "11695\n",
      "0.8907333768\n",
      "episode is 47\n",
      "Random_start:24\n",
      "11849\n",
      "0.889378091237\n",
      "episode is 48\n",
      "Random_start:22\n",
      "11961\n",
      "0.888393739115\n",
      "episode is 49\n",
      "Normal_start:1\n",
      "12391\n",
      "0.88462475516\n",
      "episode is 50\n",
      "Random_start:74\n",
      "12528\n",
      "0.883427339662\n",
      "episode is 51\n",
      "Random_start:94\n",
      "12955\n",
      "0.879705756157\n",
      "episode is 52\n",
      "Random_start:49\n",
      "13005\n",
      "0.879271011974\n",
      "episode is 53\n",
      "Random_start:22\n",
      "13086\n",
      "0.878567187542\n",
      "episode is 54\n",
      "Random_start:92\n",
      "13216\n",
      "0.877438783819\n",
      "episode is 55\n",
      "Random_start:31\n",
      "13275\n",
      "0.876927145885\n",
      "episode is 56\n",
      "Random_start:19\n",
      "13389\n",
      "0.875939412054\n",
      "episode is 57\n",
      "Random_start:97\n",
      "13409\n",
      "0.875766241489\n",
      "episode is 58\n",
      "Random_start:37\n",
      "13474\n",
      "0.875203676286\n",
      "episode is 59\n",
      "Normal_start:1\n",
      "13607\n",
      "0.874053720286\n",
      "episode is 60\n",
      "Normal_start:1\n",
      "13717\n",
      "0.873103783755\n",
      "episode is 61\n",
      "Normal_start:1\n",
      "13834\n",
      "0.872094542849\n",
      "episode is 62\n",
      "Random_start:15\n",
      "13928\n",
      "0.871284554733\n",
      "episode is 63\n",
      "Random_start:27\n",
      "13990\n",
      "0.870750723814\n",
      "episode is 64\n",
      "Random_start:22\n",
      "14038\n",
      "0.870337662609\n",
      "episode is 65\n",
      "Normal_start:1\n",
      "14182\n",
      "0.869099667945\n",
      "episode is 66\n",
      "Random_start:41\n",
      "14246\n",
      "0.868550020063\n",
      "episode is 67\n",
      "Random_start:47\n",
      "14514\n",
      "0.866252186482\n",
      "episode is 68\n",
      "Random_start:22\n",
      "14539\n",
      "0.866038150191\n",
      "episode is 69\n",
      "Normal_start:1\n",
      "14714\n",
      "0.864541393472\n",
      "episode is 70\n",
      "Random_start:34\n",
      "14916\n",
      "0.862816962119\n",
      "episode is 71\n",
      "Normal_start:1\n",
      "14964\n",
      "0.862407708206\n",
      "episode is 72\n",
      "Random_start:99\n",
      "14965\n",
      "0.862399184172\n",
      "episode is 73\n",
      "Random_start:60\n",
      "14997\n",
      "0.862126460071\n",
      "episode is 74\n",
      "Normal_start:1\n",
      "15068\n",
      "0.861521665012\n",
      "episode is 75\n",
      "Random_start:31\n",
      "15170\n",
      "0.860653555725\n",
      "episode is 76\n",
      "Random_start:42\n",
      "15218\n",
      "0.860245339998\n",
      "episode is 77\n",
      "Random_start:81\n",
      "15378\n",
      "0.858886035188\n",
      "episode is 78\n",
      "Random_start:31\n",
      "15490\n",
      "0.857935815051\n",
      "episode is 79\n",
      "Normal_start:1\n",
      "15528\n",
      "0.857613660654\n",
      "episode is 80\n",
      "Normal_start:1\n",
      "15729\n",
      "0.855911668272\n",
      "episode is 81\n",
      "Random_start:55\n",
      "15769\n",
      "0.855573371268\n",
      "episode is 82\n",
      "Random_start:25\n",
      "15826\n",
      "0.855091531784\n",
      "episode is 83\n",
      "Random_start:91\n",
      "15992\n",
      "0.853689843564\n",
      "episode is 84\n",
      "Random_start:51\n",
      "16167\n",
      "0.852214677485\n",
      "episode is 85\n",
      "Normal_start:1\n",
      "16246\n",
      "0.851549590634\n",
      "episode is 86\n",
      "Normal_start:1\n",
      "16534\n",
      "0.849129414539\n",
      "episode is 87\n",
      "Random_start:32\n",
      "16554\n",
      "0.848961605437\n",
      "episode is 88\n",
      "Normal_start:1\n",
      "16715\n",
      "0.847611964006\n",
      "episode is 89\n",
      "Random_start:11\n",
      "16857\n",
      "0.846423399098\n",
      "episode is 90\n",
      "Random_start:13\n",
      "16991\n",
      "0.845303342348\n",
      "episode is 91\n",
      "Random_start:24\n",
      "17260\n",
      "0.843059395819\n",
      "episode is 92\n",
      "Random_start:19\n",
      "17308\n",
      "0.842659623262\n",
      "episode is 93\n",
      "Random_start:58\n",
      "17323\n",
      "0.842534733685\n",
      "episode is 94\n",
      "Random_start:52\n",
      "17481\n",
      "0.841220367429\n",
      "episode is 95\n",
      "Random_start:42\n",
      "17710\n",
      "0.839319050626\n",
      "episode is 96\n",
      "Normal_start:1\n",
      "17772\n",
      "0.838805032177\n",
      "episode is 97\n",
      "Normal_start:1\n",
      "17879\n",
      "0.837918685073\n",
      "episode is 98\n",
      "Random_start:64\n",
      "18003\n",
      "0.836892702144\n",
      "episode is 99\n",
      "Random_start:50\n",
      "18032\n",
      "0.836652938028\n",
      "episode is 100\n",
      "Normal_start:1\n",
      "18119\n",
      "0.835934062728\n",
      "episode is 101\n",
      "Random_start:69\n",
      "18135\n",
      "0.835801923849\n",
      "episode is 102\n",
      "Random_start:55\n",
      "18227\n",
      "0.835042535451\n",
      "episode is 103\n",
      "Random_start:54\n",
      "18257\n",
      "0.834795059814\n",
      "episode is 104\n",
      "Normal_start:1\n",
      "18420\n",
      "0.83345173897\n",
      "episode is 105\n",
      "Random_start:53\n",
      "18454\n",
      "0.833171812969\n",
      "episode is 106\n",
      "Random_start:77\n",
      "18527\n",
      "0.832571116827\n",
      "episode is 107\n",
      "Random_start:5\n",
      "18642\n",
      "0.831625703759\n",
      "episode is 108\n",
      "Normal_start:1\n",
      "18809\n",
      "0.830254733912\n",
      "episode is 109\n",
      "Normal_start:1\n",
      "18912\n",
      "0.829410306491\n",
      "episode is 110\n",
      "Normal_start:1\n",
      "19048\n",
      "0.828296665921\n",
      "episode is 111\n",
      "Random_start:58\n",
      "19087\n",
      "0.827977592445\n",
      "episode is 112\n",
      "Random_start:31\n",
      "19234\n",
      "0.826776048735\n",
      "episode is 113\n",
      "Random_start:74\n",
      "19298\n",
      "0.826253479304\n",
      "episode is 114\n",
      "Random_start:63\n",
      "19340\n",
      "0.825910724826\n",
      "episode is 115\n",
      "Random_start:60\n",
      "19379\n",
      "0.825592581685\n",
      "episode is 116\n",
      "Random_start:61\n",
      "19444\n",
      "0.825062618764\n",
      "episode is 117\n",
      "Normal_start:1\n",
      "19592\n",
      "0.823857218305\n",
      "episode is 118\n",
      "Random_start:49\n",
      "19626\n",
      "0.823580553886\n",
      "episode is 119\n",
      "Normal_start:1\n",
      "19677\n",
      "0.823165733592\n",
      "episode is 120\n",
      "Normal_start:1\n",
      "19887\n",
      "0.821459877327\n",
      "episode is 121\n",
      "Normal_start:1\n",
      "19927\n",
      "0.821135358284\n",
      "episode is 122\n",
      "Random_start:62\n",
      "19963\n",
      "0.820843402111\n",
      "episode is 123\n",
      "Random_start:29\n",
      "19997\n",
      "0.820567762215\n",
      "episode is 124\n",
      "Normal_start:1\n",
      "20113\n",
      "0.81962804875\n",
      "episode is 125\n",
      "Random_start:79\n",
      "20126\n",
      "0.819522803945\n",
      "episode is 126\n",
      "Random_start:25\n",
      "20167\n",
      "0.819190967627\n",
      "episode is 127\n",
      "Normal_start:1\n",
      "20390\n",
      "0.817388482287\n",
      "episode is 128\n",
      "Random_start:51\n",
      "20483\n",
      "0.816637960045\n",
      "episode is 129\n",
      "Normal_start:1\n",
      "20538\n",
      "0.816194431149\n",
      "episode is 130\n",
      "Random_start:57\n",
      "20572\n",
      "0.815920371635\n",
      "episode is 131\n",
      "Random_start:92\n",
      "20680\n",
      "0.815050447477\n",
      "episode is 132\n",
      "Normal_start:1\n",
      "20744\n",
      "0.81453538003\n",
      "episode is 133\n",
      "Normal_start:1\n",
      "20812\n",
      "0.813988481938\n",
      "episode is 134\n",
      "Random_start:8\n",
      "20853\n",
      "0.813658914226\n",
      "episode is 135\n",
      "Normal_start:1\n",
      "20971\n",
      "0.812711155995\n",
      "episode is 136\n",
      "Random_start:76\n",
      "20989\n",
      "0.81256668099\n",
      "episode is 137\n",
      "Random_start:14\n",
      "21043\n",
      "0.812133411975\n",
      "episode is 138\n",
      "Random_start:76\n",
      "21154\n",
      "0.81124353786\n",
      "episode is 139\n",
      "Random_start:26\n",
      "21353\n",
      "0.80965064867\n",
      "episode is 140\n",
      "Normal_start:1\n",
      "21407\n",
      "0.809218953888\n",
      "episode is 141\n",
      "Random_start:7\n",
      "21470\n",
      "0.808715604518\n",
      "episode is 142\n",
      "Random_start:94\n",
      "21477\n",
      "0.808659696383\n",
      "episode is 143\n",
      "Normal_start:1\n",
      "21636\n",
      "0.807390836476\n",
      "episode is 144\n",
      "Normal_start:1\n",
      "21795\n",
      "0.806123992454\n",
      "episode is 145\n",
      "Normal_start:1\n",
      "21847\n",
      "0.805710115596\n",
      "episode is 146\n",
      "Random_start:92\n",
      "21880\n",
      "0.805447574579\n",
      "episode is 147\n",
      "Random_start:43\n",
      "21931\n",
      "0.805041999746\n",
      "episode is 148\n",
      "Normal_start:1\n",
      "21996\n",
      "0.804525390363\n",
      "episode is 149\n",
      "Random_start:60\n",
      "22013\n",
      "0.804390332527\n",
      "episode is 150\n",
      "Random_start:39\n",
      "22101\n",
      "0.803691576532\n",
      "episode is 151\n",
      "Normal_start:1\n",
      "22147\n",
      "0.803326562366\n",
      "episode is 152\n",
      "Random_start:65\n",
      "22162\n",
      "0.803207572306\n",
      "episode is 153\n",
      "Random_start:94\n",
      "22173\n",
      "0.803120324272\n",
      "episode is 154\n",
      "Normal_start:1\n",
      "22376\n",
      "0.801511923093\n",
      "episode is 155\n",
      "Random_start:96\n",
      "22384\n",
      "0.801448604672\n",
      "episode is 156\n",
      "Normal_start:1\n",
      "22463\n",
      "0.800823607181\n",
      "episode is 157\n",
      "Random_start:80\n",
      "22486\n",
      "0.800641738667\n",
      "episode is 158\n",
      "Normal_start:1\n",
      "22560\n",
      "0.800056880205\n",
      "episode is 159\n",
      "Random_start:44\n",
      "22674\n",
      "0.799156728545\n",
      "episode is 160\n",
      "Random_start:30\n",
      "22702\n",
      "0.798935795594\n",
      "episode is 161\n",
      "Random_start:43\n",
      "22780\n",
      "0.798320665605\n",
      "episode is 162\n",
      "Normal_start:1\n",
      "22834\n",
      "0.797895087362\n",
      "episode is 163\n",
      "Random_start:9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22867\n",
      "0.797635124879\n",
      "episode is 164\n",
      "Random_start:19\n",
      "22932\n",
      "0.7971233284\n",
      "episode is 165\n",
      "Normal_start:1\n",
      "23039\n",
      "0.796281556867\n",
      "episode is 166\n",
      "Random_start:25\n",
      "23181\n",
      "0.79516582941\n",
      "episode is 167\n",
      "Normal_start:1\n",
      "23381\n",
      "0.793597067036\n",
      "episode is 168\n",
      "Random_start:16\n",
      "23447\n",
      "0.793080063602\n",
      "episode is 169\n",
      "Normal_start:1\n",
      "23577\n",
      "0.792062720935\n",
      "episode is 170\n",
      "Random_start:61\n",
      "23647\n",
      "0.791515468591\n",
      "episode is 171\n",
      "Random_start:77\n",
      "23733\n",
      "0.79084365421\n",
      "episode is 172\n",
      "Random_start:82\n",
      "23761\n",
      "0.790625048593\n",
      "episode is 173\n",
      "Normal_start:1\n",
      "23831\n",
      "0.790078802267\n",
      "episode is 174\n",
      "Random_start:46\n",
      "23881\n",
      "0.78968886036\n",
      "episode is 175\n",
      "Random_start:91\n",
      "24008\n",
      "0.788699284021\n",
      "episode is 176\n",
      "Random_start:78\n",
      "24015\n",
      "0.788644776979\n",
      "episode is 177\n",
      "Normal_start:1\n",
      "24071\n",
      "0.788208857973\n",
      "episode is 178\n",
      "Random_start:66\n",
      "24078\n",
      "0.788154385259\n",
      "episode is 179\n",
      "Normal_start:1\n",
      "24117\n",
      "0.78785096422\n",
      "episode is 180\n",
      "Normal_start:1\n",
      "24167\n",
      "0.787462135953\n",
      "episode is 181\n",
      "Random_start:77\n",
      "24175\n",
      "0.78739994147\n",
      "episode is 182\n",
      "Normal_start:1\n",
      "24224\n",
      "0.78701910881\n",
      "episode is 183\n",
      "Random_start:52\n",
      "24311\n",
      "0.786343396163\n",
      "episode is 184\n",
      "Random_start:23\n",
      "24365\n",
      "0.7859242839\n",
      "episode is 185\n",
      "Normal_start:1\n",
      "24417\n",
      "0.785520908159\n",
      "episode is 186\n",
      "Random_start:45\n",
      "24538\n",
      "0.784583095351\n",
      "episode is 187\n",
      "Random_start:19\n",
      "24558\n",
      "0.784428194223\n",
      "episode is 188\n",
      "Normal_start:1\n",
      "24692\n",
      "0.783391155414\n",
      "episode is 189\n",
      "Normal_start:1\n",
      "24839\n",
      "0.782255105616\n",
      "episode is 190\n",
      "Random_start:30\n",
      "24956\n",
      "0.781352095507\n",
      "episode is 191\n",
      "Random_start:98\n",
      "24961\n",
      "0.781313528866\n",
      "episode is 192\n",
      "Random_start:17\n",
      "25030\n",
      "0.7807815061\n",
      "episode is 193\n",
      "Normal_start:1\n",
      "25091\n",
      "0.780311472756\n",
      "episode is 194\n",
      "Random_start:54\n",
      "25127\n",
      "0.780034210536\n",
      "episode is 195\n",
      "Random_start:7\n",
      "25182\n",
      "0.779610808167\n",
      "episode is 196\n",
      "Random_start:24\n",
      "25290\n",
      "0.778780077169\n",
      "episode is 197\n",
      "Normal_start:1\n",
      "25385\n",
      "0.778050082898\n",
      "episode is 198\n",
      "Random_start:5\n",
      "25414\n",
      "0.777827380668\n",
      "episode is 199\n",
      "Random_start:89\n",
      "25432\n",
      "0.777689184177\n",
      "episode is 200\n",
      "Random_start:88\n",
      "25457\n",
      "0.777497285869\n",
      "episode is 201\n",
      "Random_start:90\n",
      "25458\n",
      "0.777489610935\n",
      "episode is 202\n",
      "Random_start:45\n",
      "25508\n",
      "0.77710596205\n",
      "episode is 203\n",
      "Random_start:5\n",
      "25545\n",
      "0.776822185346\n",
      "episode is 204\n",
      "Normal_start:1\n",
      "25637\n",
      "0.776117033355\n",
      "episode is 205\n",
      "Random_start:34\n",
      "25668\n",
      "0.775879573883\n",
      "episode is 206\n",
      "Random_start:26\n",
      "25700\n",
      "0.775634531628\n",
      "episode is 207\n",
      "Normal_start:1\n",
      "25761\n",
      "0.775167636981\n",
      "episode is 208\n",
      "Random_start:63\n",
      "25828\n",
      "0.774655146368\n",
      "episode is 209\n",
      "Normal_start:1\n",
      "25893\n",
      "0.774158282021\n",
      "episode is 210\n",
      "Random_start:64\n",
      "25911\n",
      "0.774020745909\n",
      "episode is 211\n",
      "Normal_start:1\n",
      "25983\n",
      "0.773470848958\n",
      "episode is 212\n",
      "Normal_start:1\n",
      "26022\n",
      "0.773173153382\n",
      "episode is 213\n",
      "Random_start:74\n",
      "26056\n",
      "0.772913718616\n",
      "episode is 214\n",
      "Normal_start:1\n",
      "26131\n",
      "0.772341747843\n",
      "episode is 215\n",
      "Normal_start:1\n",
      "26177\n",
      "0.771991151282\n",
      "episode is 216\n",
      "Normal_start:1\n",
      "26287\n",
      "0.771153421851\n",
      "episode is 217\n",
      "Normal_start:1\n",
      "26453\n",
      "0.769890955308\n",
      "episode is 218\n",
      "Normal_start:1\n",
      "26536\n",
      "0.769260507487\n",
      "episode is 219\n",
      "Normal_start:1\n",
      "26618\n",
      "0.768638169065\n",
      "episode is 220\n",
      "Random_start:48\n",
      "26685\n",
      "0.76813005173\n",
      "episode is 221\n",
      "Normal_start:1\n",
      "26736\n",
      "0.767743503982\n",
      "episode is 222\n",
      "Normal_start:1\n",
      "26826\n",
      "0.767061841622\n",
      "episode is 223\n",
      "Random_start:62\n",
      "26916\n",
      "0.766380792483\n",
      "episode is 224\n",
      "Random_start:64\n",
      "26984\n",
      "0.765866628379\n",
      "episode is 225\n",
      "Random_start:46\n",
      "27041\n",
      "0.765435907168\n",
      "episode is 226\n",
      "Normal_start:1\n",
      "27175\n",
      "0.76442430098\n",
      "episode is 227\n",
      "Random_start:68\n",
      "27234\n",
      "0.763979321925\n",
      "episode is 228\n",
      "Random_start:25\n",
      "27251\n",
      "0.763851156334\n",
      "episode is 229\n",
      "Normal_start:1\n",
      "27364\n",
      "0.762999785643\n",
      "episode is 230\n",
      "Normal_start:1\n",
      "27446\n",
      "0.762382578908\n",
      "episode is 231\n",
      "Normal_start:1\n",
      "27485\n",
      "0.762089206913\n",
      "episode is 232\n",
      "Random_start:94\n",
      "27595\n",
      "0.761262363633\n",
      "episode is 233\n",
      "Random_start:20\n",
      "27676\n",
      "0.760654087503\n",
      "episode is 234\n",
      "Random_start:44\n",
      "27724\n",
      "0.760293860003\n",
      "episode is 235\n",
      "Random_start:5\n",
      "27778\n",
      "0.759888810692\n",
      "episode is 236\n",
      "Normal_start:1\n",
      "27946\n",
      "0.75863005514\n",
      "episode is 237\n",
      "Random_start:48\n",
      "28012\n",
      "0.75813612232\n",
      "episode is 238\n",
      "Random_start:5\n",
      "28100\n",
      "0.757478052125\n",
      "episode is 239\n",
      "Normal_start:1\n",
      "28164\n",
      "0.756999819223\n",
      "episode is 240\n",
      "Normal_start:1\n",
      "28191\n",
      "0.756798156497\n",
      "episode is 241\n",
      "Random_start:42\n",
      "28296\n",
      "0.756014429962\n",
      "episode is 242\n",
      "Normal_start:1\n",
      "28335\n",
      "0.755723541061\n",
      "episode is 243\n",
      "Random_start:75\n",
      "28387\n",
      "0.755335865624\n",
      "episode is 244\n",
      "Random_start:19\n",
      "28439\n",
      "0.754948391726\n",
      "episode is 245\n",
      "Normal_start:1\n",
      "28470\n",
      "0.754717493515\n",
      "episode is 246\n",
      "Normal_start:1\n",
      "28554\n",
      "0.754092193484\n",
      "episode is 247\n",
      "Random_start:86\n",
      "28562\n",
      "0.754032668489\n",
      "episode is 248\n",
      "Random_start:50\n",
      "28611\n",
      "0.753668181788\n",
      "episode is 249\n",
      "Random_start:23\n",
      "28691\n",
      "0.753073485153\n",
      "episode is 250\n",
      "Random_start:96\n",
      "28746\n",
      "0.752664907105\n",
      "episode is 251\n",
      "Normal_start:1\n",
      "28787\n",
      "0.752360476906\n",
      "episode is 252\n",
      "Random_start:40\n",
      "28849\n",
      "0.751900356062\n",
      "episode is 253\n",
      "Normal_start:1\n",
      "28916\n",
      "0.751403449306\n",
      "episode is 254\n",
      "Random_start:99\n",
      "28917\n",
      "0.751396035309\n",
      "episode is 255\n",
      "Random_start:87\n",
      "28967\n",
      "0.75102542995\n",
      "episode is 256\n",
      "Normal_start:1\n",
      "29058\n",
      "0.750351403538\n",
      "episode is 257\n",
      "Normal_start:1\n",
      "29100\n",
      "0.750040521238\n",
      "episode is 258\n",
      "Normal_start:1\n",
      "29221\n",
      "0.749145613735\n",
      "episode is 259\n",
      "Normal_start:1\n",
      "29267\n",
      "0.748805684943\n",
      "episode is 260\n",
      "Normal_start:1\n",
      "29306\n",
      "0.748517606904\n",
      "episode is 261\n",
      "Random_start:26\n",
      "29373\n",
      "0.748022965831\n",
      "episode is 262\n",
      "Random_start:9\n",
      "29407\n",
      "0.747772080676\n",
      "episode is 263\n",
      "Normal_start:1\n",
      "29459\n",
      "0.747388538923\n",
      "episode is 264\n",
      "Random_start:26\n",
      "29484\n",
      "0.74720421483\n",
      "episode is 265\n",
      "Random_start:58\n",
      "29584\n",
      "0.746467379094\n",
      "episode is 266\n",
      "Random_start:85\n",
      "29652\n",
      "0.745966751509\n",
      "episode is 267\n",
      "Random_start:97\n",
      "29779\n",
      "0.745032667004\n",
      "episode is 268\n",
      "Random_start:93\n",
      "29888\n",
      "0.744231917885\n",
      "episode is 269\n",
      "Normal_start:1\n",
      "29991\n",
      "0.743476048349\n",
      "episode is 270\n",
      "Normal_start:1\n",
      "30050\n",
      "0.743043425117\n",
      "episode is 271\n",
      "Random_start:62\n",
      "30077\n",
      "0.742845530109\n",
      "episode is 272\n",
      "Random_start:33\n",
      "30143\n",
      "0.742362011638\n",
      "episode is 273\n",
      "Normal_start:1\n",
      "30217\n",
      "0.74182026422\n",
      "episode is 274\n",
      "Random_start:56\n",
      "30241\n",
      "0.741644648432\n",
      "episode is 275\n",
      "Normal_start:1\n",
      "30265\n",
      "0.741469074786\n",
      "episode is 276\n",
      "Random_start:23\n",
      "30327\n",
      "0.741015704519\n",
      "episode is 277\n",
      "Random_start:33\n",
      "30426\n",
      "0.740292357087\n",
      "episode is 278\n",
      "Random_start:30\n",
      "30479\n",
      "0.739905404689\n",
      "episode is 279\n",
      "Random_start:84\n",
      "30534\n",
      "0.739504067095\n",
      "episode is 280\n",
      "Random_start:42\n",
      "30715\n",
      "0.738184858977\n",
      "episode is 281\n",
      "Random_start:67\n",
      "30751\n",
      "0.737922759608\n",
      "episode is 282\n",
      "Random_start:73\n",
      "30856\n",
      "0.737158841838\n",
      "episode is 283\n",
      "Random_start:14\n",
      "30881\n",
      "0.736977074849\n",
      "episode is 284\n",
      "Random_start:94\n",
      "30896\n",
      "0.736868036466\n",
      "episode is 285\n",
      "Normal_start:1\n",
      "30954\n",
      "0.73644657524\n",
      "episode is 286\n",
      "Normal_start:1\n",
      "30992\n",
      "0.736170577985\n",
      "episode is 287\n",
      "Normal_start:1\n",
      "31029\n",
      "0.735901944571\n",
      "episode is 288\n",
      "Random_start:19\n",
      "31054\n",
      "0.735720491767\n",
      "episode is 289\n",
      "Random_start:21\n",
      "31195\n",
      "0.734697946938\n",
      "episode is 290\n",
      "Random_start:69\n",
      "31219\n",
      "0.7345240403\n",
      "episode is 291\n",
      "Random_start:5\n",
      "31330\n",
      "0.733720264793\n",
      "episode is 292\n",
      "Normal_start:1\n",
      "31473\n",
      "0.73268608443\n",
      "episode is 293\n",
      "Random_start:83\n",
      "31523\n",
      "0.732324831708\n",
      "episode is 294\n",
      "Random_start:50\n",
      "31565\n",
      "0.732021518979\n",
      "episode is 295\n",
      "Normal_start:1\n",
      "31607\n",
      "0.731718333614\n",
      "episode is 296\n",
      "Normal_start:1\n",
      "31682\n",
      "0.731177247797\n",
      "episode is 297\n",
      "Random_start:98\n",
      "31695\n",
      "0.731083500848\n",
      "episode is 298\n",
      "Random_start:96\n",
      "31705\n",
      "0.731011396103\n",
      "episode is 299\n",
      "Random_start:81\n",
      "31750\n",
      "0.730687013967\n",
      "11\n",
      "0.5 0.5\n",
      "0.5 1.5\n",
      "12\n",
      "0.5 1.5\n",
      "1.5 1.5\n",
      "22\n",
      "1.5 1.5\n",
      "1.5 2.5\n",
      "32\n",
      "1.5 2.5\n",
      "1.5 3.5\n",
      "42\n",
      "1.5 3.5\n",
      "1.5 4.5\n",
      "43\n",
      "1.5 4.5\n",
      "2.5 4.5\n",
      "44\n",
      "2.5 4.5\n",
      "3.5 4.5\n",
      "45\n",
      "3.5 4.5\n",
      "4.5 4.5\n",
      "46\n",
      "4.5 4.5\n",
      "5.5 4.5\n",
      "47\n",
      "5.5 4.5\n",
      "6.5 4.5\n",
      "48\n",
      "6.5 4.5\n",
      "7.5 4.5\n",
      "58\n",
      "7.5 4.5\n",
      "7.5 5.5\n",
      "68\n",
      "7.5 5.5\n",
      "7.5 6.5\n",
      "69\n",
      "7.5 6.5\n",
      "8.5 6.5\n",
      "70\n",
      "8.5 6.5\n",
      "9.5 6.5\n",
      "80\n",
      "9.5 6.5\n",
      "9.5 7.5\n",
      "90\n",
      "9.5 7.5\n",
      "9.5 8.5\n",
      "100\n",
      "9.5 8.5\n",
      "9.5 9.5\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X2QVPWd7/H3d3hyEBdoM1EZdfEmwWg0gg8hyI0wGnfz\nYAIVtxK91/hQuZds4gaz6+IKtyqZ3VurriTZlapcvVbENWVCMisK0b2aZNUxiZJZhRkfkJgYxwdA\nhsRmNMQphnG+94/uHk43o9B9Duf8pvvzqpqa6dPT53w4NB9+85vu8zN3R0RE6ktT1gFERCR5KncR\nkTqkchcRqUMqdxGROqRyFxGpQyp3EZE6dMByN7PVZrbTzJ6JbMuZ2U/N7DfFz9MPbUwREanGwYzc\n/xX4WMW2a4EH3f19wIPF2yIiEgg7mDcxmdlM4D53P6V4+zlgobu/ambHAJ3ufuKhDCoiIgdvfI2P\nO8rdXy1+vQM46u2+0cyWAEsADjvssDOOP/74Gg+ZjOHhYZqasv9VQwg5QsgQSo4QMoSSI4QMoeQI\nIQPAr3/969+7e0tVD3L3A34AM4FnIrf7K+7fdTD7mTVrlmft4YcfzjqCu4eRI4QM7mHkCCGDexg5\nQsjgHkaOEDK4uwNP+EF0bPSj1v+S+orTMRQ/76xxPyIicgjUWu4/Ai4rfn0ZsD6ZOCIikoSDeSnk\nGmADcKKZbTWzLwA3AOeb2W+AjxZvi4hIIA74C1V3v/ht7jov4SwiIpKQ7H8NLCIiiVO5i0hi8vn8\nO96W9KjcRSQRRz/cw8lPvjxS6Pl8npOffJmjH+7JOFlGnuqAfz4F2qcVPj/VkerhVe4iElt0hF4q\n+JOffHnU+xvCUx1w71J4/RXAC5/vXZpqwavcRSS2XC7Hs6fte/d5tNifPe14crlcFrGy8+A/wN4B\n7mcB97OgsG3vQGF7SlTuIpKIyoKHBi12gNe3ArCDFnbQst/2NKjcRSQRlVMxQNkcfEOZemx12w8B\nlbuIxFZZ7JVTNA1X8Od9DSY0l2+b0FzYnhKVu4jEFp16KU3FRAu+4aZmPvhZ+NQqGHcYYDD1uMLt\nD342tQi1XvJXRKTMjrbZ5PP5kSIvFHwDFnvJBz8LG/9Y+PqK76R+eI3cRSQxlUXesMUeAJW7iEgd\nUrmLiNShWOVuZleZ2TNmttnMvppUKBERiafmcjezU4D/CXwIOA24wMzem1QwERGpXZyR+0lAl7u/\n6e5DwCPAZ5KJJSIicVhh7dUaHmh2EoXl9eYBA8CDFBZx/UrF9y0BlgC0tLSc0dGR7pXRKu3evZsp\nU6ZkmiGUHCFkCCVHCBlCyRFChlByxM3Q3d0NwJw5c2LlaGtr2+juZ1b1oGpX1I5+AF8ANgI/A24G\n/uWdvn/WrFmHbnnwgxTKauYh5Aghg3sYOULI4B5GjhAyuIeRI26G1atX++rVq2PnoDBwrqqfY/1C\n1d1vc/cz3P0cYBfw6zj7ExGRZMR6h6qZvdvdd5rZ8RTm2z+cTCwREYkj7uUH1prZkcBe4Ep3708g\nk4iIxBSr3N39I0kFERGR5OgdqiIidWjMlLtWVZdQ6bkpIRoT5d7e3s6qVavKVlVftWoV7e3t2QaT\nhvfgQ++hu+essudmd89ZPPjQezJOJllbt34t3b19dPW+xvwVd7Ju/dpUjx98uUdHQaWCX7Vq1aj3\ni6Qp+twrFXx3z1mj3i+NZd36tSzfYAwyATC2DU9n+QZLteCDL/dcLsfSpUtHbkeLfenSpbpetGQm\nl8sxZ/bjI7ejxT5n9uN6bjawlV0DDDCJVoxWDIABJrGyayC1DMGXO+xf8KBilzBUFjyo2AW2D08D\n4MjhIzhy+IjI9qmpZRgT5V45FQOUzcGLZKVyKgYom4OXxjSjqfCWn+OG3stxQ++NbH89tQzBl3tl\nsVdO0egfkWSlstgrp2j03Gxcy+Y208yesm3N7GHZ3ObUMgRf7tEfb0tTMdGC14+/kpXoc680FRMt\neD03G9fiRRdy/TxnIkMAtDbt4vp5zuJFF6aWIe7lB1LR3t6+36rqmnOXEJx37m/3e25qzl2gUPA7\ndzwFwKNfbEv9+MGP3Eu0qrqESs9NCdGYKXcRETl4KncRkToUq9zN7K/NbLOZPWNma8zssKSCiYhI\n7WoudzNrBZYCZ7r7KcA44KKkgomISO3iTsuMB5rNbDwwGdgeP5KIiMRlhbVXa3yw2VXAPwIDwE/c\n/b+P8j1LgCUALS0tZ3R0dNR8vCSEsKJ6KDlCyBBKjhAyhJIjhAyh5IibobWrMH7eNnc4Vo62traN\n7n5mVQ+qdkXt0gcwHXgIaAEmAOuAS97pMbNmzYq9CnhcIayo7h5GjhAyuIeRI4QM7mHkCCGDexg5\n4mbou+VJ77vlydg5gCe8yo6OMy3zUaDX3X/n7nuBu4GzY+xPREQSEqfcXwY+bGaTzcyA84AtycQS\nEZE4ai53d+8C7gI2AU8X93VrQrlERCSGWNeWcfevA19PKIuIiCRE71AVEalDKncRSUzlNex1Tfvs\nqNxFJBFbr/05b964eaTQ8/k8b964ma3X/jzjZNlYt34t3b19dPW+xvwVd6a6ODao3EUkAdEReqng\n37xx86j3N4J169eyfIMxWPy15rbh6SzfYKkWvMpdRGLL5XJMvuYDI7ejxT75mg803DXuV3YNMMAk\npu59nal7C+umDjCJlV0DqWVQuYtIIioLHhqz2AG2D08DYNfgTnYN7oxsn5paBpW7iCSicioGKJuD\nbyQzmvoB6HzjCTrfeCKy/fXUMqjcRSS2ymKvnKJptIJfNreZZvaUbWtmD8vmNqeWQeUuIrFFp15K\nUzHRgm+0qZnFiy7k+nnOJPZiOK1Nu7h+nrN40YWpZYj1DlURkZJjb/gI+Xx+pMhzuRw06Jw7FAp+\nT8/jAHzr65ekfnyN3EUkMZVF3qjFHgKVu4hIHYqzhuqJZtYT+XjDzL6aZDgREalNzXPu7v4cMBvA\nzMYB24B7EsolIiIxJDUtcx7wW3d/KaH9iYhIDEmV+0XAmoT2JSIiMVlh7dUYOzCbCGwHPuDufaPc\nvwRYAtDS0nJGR0dHrOPFFcKK6qHkCCFDKDlCyBBKjhAyhJIjbobn1v8AgBMXXRQrR1tb20Z3P7Oq\nB1W7onblB7AI+MnBfO+sWbNirwIeVwgrqruHkSOEDO5h5Aghg3sYOULI4B5GjrgZftD+d/6D9r+L\nnQN4wqvs5iSmZS5GUzIiIkGJVe5mdjhwPnB3MnFERCQJcRfI/iNwZEJZREQkIXqHqohIHVK5i8QU\nyqLQoeSQMOiqkCIxfPsvHwLg4utmk8vlyOfzrFnRA8CVt5ybWo5vfu4CAK64+bsjOW7/0qUAXP3D\n+1LLsW79WlZ2DbB9eBozmvpZNrc51cvchpRj3fq1dL/QxyDjWbXiztQzaOQuUqPoyHjNip6yYq+8\nP60ct3/p0rJiTzNHaVHobcPTcSyTRaFDyVHKsIcJmWXQyF2kRrlcjouvmz1S6NFiL43k08pxxc3f\nHSn0aLGXRvJpKCwKPZ0vP//PnPDaboYHi6+1+MUQL921vur9Te/v56XbVlcfpLePdh/P7mMWsnvK\ncWw//EhgIpt+OoQ/v6mqXfX3D7NrY3WPAejuncynfTzThoy9VthWWCB7F4sXVb27mmjkLhJDqeCj\n0iz2aI4rbv5u2bY0ix32LQpt4/8ITYMj2/d4umPItzveYIo5SscaGt/CwISWke1pLpCtkbtIDJVT\nMVAYwadd8JVTMVAYwadZ8DOa+tk2PJ3V5xTKbODlLwLQ2rSLz11X/UpEvZ2dnLZwYdWP+28r7mTb\n8PTIlsGRHDdcfX5V++rs7GThwtOrzvCNUoYj5pVt1wLZImNAZbFHR/ClOfi0clROxZSU5uDTEMKi\n0KHkCCGDyl2kRtERcWmkHi34NOfcS0oj9WjBp5VjZFFoG8Igk0Whozlam3ZhDGeSI4QMmpYRieHK\nW87db1HoLObcr/7hffvlSHvOHQqltv6Bwksvb/9y+otCR3Ok9YvLUDNo5C4SUyiLQoeSQ8KgchcR\nqUMqdxGROhT3kr/TzOwuM/uVmW0xs3kHfpSIiBxqcX+hehPwgLv/RXG5vckJZBIRkZhqLnczmwqc\nA1wO4O6DlN4tICIimap5gWwzmw3cCjwLnAZsBK4qLuAR/T4tkB1ojhAyhJIjhAyh5Iib4aYdNwFw\n1dFXZZojCSFkgJQXyAbOBIaAucXbNwH/+50eowWy9wkhRwgZ3MPIEUIG9zByxM1w+f2X++X3X555\njiSEkME9/QWytwJb3b2rePsuoPqLMIiISOJqLnd33wG8YmYnFjedR2GKRkREMhb31TJfAb5XfKXM\nC8AV8SOJiEhcscrd3XsozL2LiEhA9A5VEZE6pHIXianyeulpXT891BwSBpW7SAyn3nEqC+5dMFKk\n+XyeBfcu4NQ7Tk01x5b3n0Tf2fPLcvSdPZ8t7z8p1Rzr1q+lu7ePrhdeY/6KO1NfHFv2UbmL1Cg6\nMi4V/IJ7F4x6f1o5SgXfd/b81HOsW7+W5RuMPT4eB7YNT2f5BlPBZ0TlLlKjXC7HI596ZOR2tNgf\n+dQjqa7EdNRjj47cjhb7UY89mlqOlV0DDDCJmdbHTOsDYIBJrOwaSOX4Uk7lLhJDZcFDusUezREt\neEi32AG2D08D4MTBQU4cHIxsn5paBtlH5S4SQ+VUDFA2B59mjuiIHSibg0/DjKZ+AC56bRIXvTYp\nsv311DLIPip3kRpVFnvlFE2ac+6VUzElaRb8srnNNLOnbFsze1g2tzmV40s5lbtIjaJTHqWpmGjB\npznnXlKaiokWfFo5Fi+6kOvnORMZApzWpl1cP89ZvOjCVI4v5eJefkCkoT192dPk8/mRAi0VfNpz\n7if9ast+OUh5zh0KBU9+NQCPXnFJqseWchq5i8RUWaBpF2poOSQMKncRkTqkchcRqUOx5tzN7EXg\nD8BbwJBXuwyUiIgcEkn8QrXN3X+fwH5ERCQhmpYREalDVlh7tcYHm/UCr1OYlvm/7n7rKN+zBFgC\n0NLSckZHR0fNx0tCKKuZh5AjhAyh5AghQyg54maY3f2/AOiZ84+Z5khCCBkA2traNlY97V3titrR\nD6C1+PndwJPAOe/0/bNmzTpEa4MfvFBWMw8hRwgZ3MPIEUIG9zByxM6w+hOFj6xzJCCEDO7uwBNe\nZT/HmpZx923FzzuBe4APxdmfiIgko+ZyN7PDzeyI0tfAnwHPJBVMRERqF+fVMkcB95hZaT/fd/cH\nEkklIiKx1Fzu7v4CcFqCWUREJCF6KaSISB1SuYuI1CFd8rcK69avZWXXANuHpzGjqZ9lc5szuVZ1\nCDlCyBBKjhAyhJJj3fq1HN/bxyDjuXrFnZmdC9HI/aCVVnbfNjwdxzJb2T2EHCFkCCVHCBlCyVHK\nMFgcM2Z1LqRAI/eDVFjZfToX922gee/v2DWu8M7eX969lz09j1e9v/7+fvoeqf7FRd0v9PExJrDw\nT85k+sR38/qEqcBkJm4YYueOp6raV2t/Ezufq+4xAM29f8KNjOeV8c/zWtMf2EbhXDy0cS+78rdX\nvb/+/n56e3urflx3bx8LJk7gtPf+gimH53nDDwPg90NDbNxUXaG8NdzPxk37vcH6gH4/1MdfnTme\nH0/+FC/ZTHa+2cIe4KtDQ9zR/Zuq99fvU7iphsd1Dx3NnrPGM+7l3TS9sZemgbcYYBLXbNjLmh0b\nqsvQP8DNz1X3GIDu3nEMMqFs2wCTWNm1i8WLqt6dxKRyP0illd0nD8P4yBUbBlM+hW93vDRzhJDh\nHXP4uPQy+Nudi/QyFI6X/d9J6VjPDv9p2fbtw1NTyyD7qNwP0oymfrYNT2f3lMKCyPccMQhAa9Mu\nvvX16pcT6+zsZOHChVU/btWKO9k2PJ17Rra8OZLj0S+2VbWvZzs7ed/CD1ad4YZiBoZmlG1vbdrF\nLTUsrVbrufhOMccDz75/vxxf/ovqcnR2dnLG6dVnWHpX8VwUTeL3IxnuufDDVe+vs7OThXOqv3L2\n/H8rz1HS2tTPD794QfUZFs6rPkPx7+Mfhi4t2z6j6fWq9yXxac79IIWysnsIOULIEEqOEDKEkiOE\nDLKPyv0glVZ2n2RDGGS2snspR2vTLozhTHKEkCGUHCFkCCVHCBlkH03LVGHxogvx5zcBcMPV52ea\nI+tfUIWQIZQcIWQIJUcIGaRAI3cRkTqkchcRqUMqdxGROhS73M1snJl1m9l9SQQSEZH4khi5XwVs\nSWA/IiKSkFjlbmbHAp8EvpNMHBERSYIV1l6t8cFmdwHXA0cAf+vu+70VzsyWAEsAWlpazujo6Kj5\neEmIu5p574PDAJxwXrwfekJYVT2EDKHkCCFDKDlCyBBKjhAyALS1tW109+reulztitqlD+AC4P8U\nv14I3Hegx8yaNetQLQ5+0OKuZn73Nzb63d/YmHmOJISQwT2MHCFkcA8jRwgZ3MPIEUIGd3fgCa+y\no+MMP+cDnzazF4EfAOea2Z0x9iciIgmpudzdfbm7H+vuM4GLgIfcvfqrRomISOL0OncRkTqUyLVl\n3L0T6ExiXyIiEp9G7iIidWjMlHs+n3/H240kiHNRecwG/vsQCdGYKPejH+7h5CdfHimxfD7PyU++\nzNEP96SaY936tXT39tH1wmvMX3FnJgv/zrz23zn9xg1l5+L0Gzcw89p/Ty9E+1RYdcK+Qs/nC7fb\ntZyaSCiCL/foqLRU8Cc/+fKo9x9KpZXd9/h4nGxWdo/+WUsFf/qNG0a9/xCG2Pd1qeBXnTD6/SKS\nmeDLPZfL8expx4/cjhb7s6cdTy6XSyXHyq4BBpjEjD++xow/vgaUVnYfSOX4UDgXm67Zt7ZltNg3\nXTMvnXORy8HS3n23o8W+tLdwv4hkLvhyh/0LHtItdoDtw9MAmLL7FabsfiWyPd2piMqChxSLfV+I\n8oIHFbtIYMZEuVdOxQBlc/BpmNHUD8CUVzuZ8mpnZHu6K7tXTsUAZXPwKYUoH7FD+Ry8iGQu+HKv\nLPbKKZq0Si2Eld0ri71yiia1OffKqZgSFbxIMIIv9+h0Q2kqJlrwaU1HlFZ2n2RDAJms7B79s5am\nYqIFn9qce0lpKiZa8JqaEQlCIu9QPdR2tM0mn8+PlFeh4NMr9pLFiy7kpbvWA/C567K5jM6LN3xy\nv3OR+px7++uFEXrpmKWCV7GLBCP4kXtJZXmlXewhCeJcVB6zgf8+REI0ZspdREQOXs3lbmaHmdl/\nmtmTZrbZzP4+yWAiIlK7OHPue4Bz3X23mU0AfmFm97v7LxPKJiIiNaq53ItLP+0u3pxQ/Kh9QVYR\nEUlMrDl3MxtnZj3ATuCn7t6VTCwREYnDCgPwmDsxmwbcA3zF3Z+puG8JsASgpaXljI6OjtjHiyPu\naubTv/ktAHZd/TeZ5khCCBlCyRFChlByhJAhlBwhZABoa2vb6O5nVvWgalfUfrsP4GvA377T98ya\nNSvpRcGrFnc18xcv+by/eMnnM8+RhBAyuIeRI4QM7mHkCCGDexg5Qsjg7g484VV2cpxXy7QUR+yY\nWTNwPvCrWvcnIiLJifNqmWOAO8xsHIW5+w53vy+ZWCIiEkecV8s8BcxJMIuIiCRE71AVEalDY6bc\ng1gUOhA6FzIaPS/20bkYI+X+zc9dwO1furRsUejbv3Qp3/zcBanmKC2Q/csMF8jeeu3PefPGzWXn\n4s0bN7P12p+nnkXC0d7ezqpVq8qeF6tWraK9vT3bYBl48KH30N1zVtm56O45iwcfek/GydIVfLlH\n/8ctFfztX7p01PsPpegC2ZD9Atmlgn/zxs2j3i+NI/r3Xir4VatWjXp/vYv+WUsF391z1qj317vg\nyz2Xy3HFzd8duR0t9itu/m7qC2Q3TXyNponZLZA9+ZoPjNyOFvvkaz7Q0JdBbmS5XI6lS5eO3I4W\n+9KlSxvqeZHL5Zgz+/GR29FinzP78YY6F8GXO+xf8JBuscO+BbJfPMp48SiLbE9/gexowYOKXfYv\neGi8Yi+pLHhovGKHMVLulVMxQNkcfBpKC2SvPqeF1ee0RLanv0B2dMQOlM3BS2OqnIoByubgG0nl\nVAxQNgffKIIv98pir5yiabQFsiunYkpU8I2rstgrp2ga6XlRWeyVUzSNdC6CL/foj1KlqZhowWex\nQLaR/QLZpamYaME32o+dUhD9ey9NxUQLvpGeF9E/a2kqJlrwjXQuxsQC2Vf/8L79FoVOe84dCgW/\n/oHCFRZu/3I2C2Qfe8NH9jsXaM694bW3t+/3vGjUOffzzv3tfudCc+4BC2JR6EDoXMho9LzYR+di\nDJW7iIgcPJW7iEgdinM99+PM7GEze9bMNpvZVUkGExGR2sX5heoQcLW7bzKzI4CNZvZTd382oWwi\nIlKjmkfu7v6qu28qfv0HYAvQmlQwERGpXVILZM8Efgac4u5vVNxXVwtk37TjJgCuOjreLFQIC++G\nkCGUHCFkCCVHCBlCyRFCBshogWxgCrAR+MyBvrceFsi+/P7L/fL7L888RxJCyOAeRo4QMriHkSOE\nDO5h5Aghg3vKC2QDmNkEYC3wPXe/O86+REQkOXFeLWPAbcAWd/9WcpFERCSuOCP3+cDngXPNrKf4\n8YmEcomISAw1vxTS3X8B2AG/UUREUqd3qIqI1CGVu0hMldcIb6Rrhku4xsQlf6GwQPXKrgG2D09j\nRlM/y+Y2p3ot9VKG7u19DPp45q+4M5MMEpZv/+VDAFx83WxyuRz5fJ41K3oAuPKWc7OMJg1uTIzc\n161fy/INxrbh6TiFz8s3GOvWr009wx4fj0MmGSQs0RH6mhU9ZcVeeb9I2sbEyH1l1wADTOdr47/L\nyU0vjWyfuHEI8qur2tfs/n7onVZ1huN7+7h94nhWNg3wph/GFmCASazs2sXiRVXvTupALpfj4utm\njxR6tNhLI3mRrIyJkfv24dHLeJBxqWUYLP4/eNyeJlr2HDayffvw1NQySHhKBR+lYpcQjImR+4ym\nfrYNT+cfhi4t297atItHr6huubuezk4WLlxYdYarV9zJtuHp8Gpltter3pfUj8qpGCiM4FXwkrUx\nMXJfNreZZvaUbWtmD8vmNjdUBglLZbFHR/ClOXiRrIyJcl+86EKun+dMZC/gtDbt4vp5nuorVUoZ\nWpt2YQxnkkHCEh2Zl0bq0YLXyF2yNCamZaBQrmt2bADgh1+8ILMM+uWpRF15y7nk8/mRIi8VvIpd\nsjYmRu4iIasschW7hEDlLiJSh1TuIiJ1KO5iHavNbKeZPZNUIBERiS/uyP1fgY8lkENERBIUq9zd\n/WeAXswrIhIYK6y9GmMHZjOB+9z9lLe5fwmwBKClpeWMjo6Omo91fdcAAMtjvHEolNXMQ8gRQoZQ\ncoSQIZQcIWQIJUcIGQDa2to2uvuZVT2o2hW1Kz+AmcAzB/O9s2bNirUC+Gdvecw/e8tjsfYRymrm\nIeQIIYN7GDlCyOAeRo4QMriHkSOEDO7uwBNeZTfr1TIiInVI5S4iUofivhRyDbABONHMtprZF5KJ\nJSIiccS6toy7X5xUEBERSY6mZURE6pDKfQyqvE54I183XOdCZHRjptzXdW+j++V+unrzzL/hIdZ1\nb8s6UiZOveNUFty7YKTE8vk8C+5dwKl3nJpxsvRtef9J9J09v+xc9J09ny3vPynjZCLZGxPlvq57\nG8vvfprBt4YB2NY/wPK7n264go+OSksFv+DeBaPeX++if9ZSwfedPX/U+0Ua0Zgo95U/fo6BvW+V\nbRvY+xYrf/xcRomykcvleORTj4zcjhb7I596pKGuI57L5TjqsUdHbkeL/ajHHm2ocyEymjFR7tv7\nB6raXs8qCx4ar9hLKgseVOwiJWOi3GdMG/1aMm+3vZ5VTsUAZXPwjaRyKgYom4MXaWRjotyX/fmJ\nNE8YV7atecI4lv35iRklykZlsVdO0TRSqVUWe+UUTSOdC5HRjIlyXzynles/cyqt05oxoHVaM9d/\n5lQWz2nNOlqqotMNpamYaME30nRE9M9amoqJFnwjnQuR0cR6h2qaFs9pbbgyH83Tlz1NPp8fKa9S\nwTdimZ30qy37nQs05y4CjJGRu5SrLK9GLjOdC5HRqdxFROqQyl1EpA7FveTvx8zsOTN73syuTSqU\niIjEU3O5m9k44NvAx4GTgYvN7OSkgomISO3ijNw/BDzv7i+4+yDwA2BRMrFERCSOOC+FbAVeidze\nCsyt/CYzWwIsKd7cY2bPxDhmEt4F/D7jDBBGjhAyQBg5QsgAYeQIIQOEkSOEDABVv2PzkL/O3d1v\nBW4FMLMn3P3MQ33MdxJChlByhJAhlBwhZAglRwgZQskRQoZSjmofE2daZhtwXOT2scVtIiKSsTjl\n/jjwPjM7wcwmAhcBP0omloiIxFHztIy7D5nZXwE/BsYBq9198wEedmutx0tQCBkgjBwhZIAwcoSQ\nAcLIEUIGCCNHCBmghhzm7ociiIiIZEjvUBURqUMqdxGROpRKuYdwmQIzW21mO7N8nb2ZHWdmD5vZ\ns2a22cyuyijHYWb2n2b2ZDHH32eRo5hlnJl1m9l9GWZ40cyeNrOeWl5yllCGaWZ2l5n9ysy2mNm8\nDDKcWDwHpY83zOyrGeT46+Lz8hkzW2Nmh6WdoZjjqmKGzWmeh9G6ysxyZvZTM/tN8fP0A+7I3Q/p\nB4Vftv4W+C/AROBJ4ORDfdxRcpwDnA48k/axIxmOAU4vfn0E8OuMzoUBU4pfTwC6gA9ndE7+Bvg+\ncF+Gfy8vAu/K6vjFDHcA/6P49URgWsZ5xgE7gD9N+bitQC/QXLzdAVyewZ//FOAZYDKFF578B/De\nlI69X1cBNwLXFr++FvinA+0njZF7EJcpcPefAZmuvebur7r7puLXfwC2UHgyp53D3X138eaE4kfq\nv1k3s2OBTwLfSfvYITGzqRT+Qd8G4O6D7t6fbSrOA37r7i9lcOzxQLOZjadQrtszyHAS0OXub7r7\nEPAI8Jk0Dvw2XbWIwgCA4ufFB9pPGuU+2mUKGn5JJTObCcyhMGrO4vjjzKwH2An81N2zyPEvwDXA\ncAbHjnKvA0J0AAACb0lEQVTgP8xsY/FyGWk7AfgdcHtxiuo7ZnZ4BjmiLgLWpH1Qd98GfAN4GXgV\neN3df5J2Dgqj9o+Y2ZFmNhn4BOVv2kzbUe7+avHrHcBRB3qAfqGaATObAqwFvurub2SRwd3fcvfZ\nFN5Z/CEzOyXN45vZBcBOd9+Y5nHfxn8tnouPA1ea2TkpH388hR/Db3b3OcAfKfzonYnimxI/Dfxb\nBseeTmGUegIwAzjczC5JO4e7bwH+CfgJ8ADQA7yVdo7ReGFu5oA/aadR7rpMQYSZTaBQ7N9z97uz\nzlP88f9h4GMpH3o+8Gkze5HCVN25ZnZnyhmAkdEi7r4TuIfCVGKatgJbIz893UWh7LPycWCTu/dl\ncOyPAr3u/jt33wvcDZydQQ7c/TZ3P8PdzwF2UfgdWVb6zOwYgOLnnQd6QBrlrssUFJmZUZhX3eLu\n38owR4uZTSt+3QycD/wqzQzuvtzdj3X3mRSeEw+5e+ojNDM73MyOKH0N/BmFH8lT4+47gFfMrHTl\nv/OAZ9PMUOFiMpiSKXoZ+LCZTS7+ezmPwu+mUmdm7y5+Pp7CfPv3s8hR9CPgsuLXlwHrD/SANK4K\nWctlChJnZmuAhcC7zGwr8HV3vy3lGPOBzwNPF+e7AVa4+/9LOccxwB3FBVeagA53z+yliBk7Crin\n0COMB77v7g9kkOMrwPeKA6AXgCsyyFD6D+584ItZHN/du8zsLmATMAR0k90lANaa2ZHAXuDKtH7J\nPVpXATcAHWb2BeAl4LMH3E/xpTUiIlJH9AtVEZE6pHIXEalDKncRkTqkchcRqUMqdxGROqRyFxGp\nQyp3EZE69P8BeRputkb/BukAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x21bc4978198>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "Alpha=0.1 #学习速率\n",
    "Beta=0.9 #对未来奖励的衰减度\n",
    "Epsilon_start=1#选择随机动作的概率 Epsilon\n",
    "Epsilon_stop=0.01\n",
    "decay_rate=0.00001\n",
    "N_Mesh=10\n",
    "N_States=100#假设网格为5*5\n",
    "Actions=['left','right','up','down']\n",
    "Obstable=[3,5,16,21,23,27,29,31,33,34,56,57,66,67,78,81,86,98]\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Caculate_coordinate(State):\n",
    "    X_Axis=0.5+(State-1)%N_Mesh\n",
    "    Y_Axis=0.5+((State-1)//N_Mesh)\n",
    "    return X_Axis,Y_Axis\n",
    "def Initial_Q_Table (N_States,Actions):#以状态数和动作为变量，返回Q表\n",
    "    Q_Table=np.zeros([N_States,len(Actions)])#初始化Q_表\n",
    "    Q_Table=pd.DataFrame(Q_Table,index=np.arange(1,N_States+1),columns=Actions)#Q_表定义行列标签\n",
    "    print ('初始化Q表完成') \n",
    "    return Q_Table\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Select_Action(State,Q_Table,ep):#根据目前的状态选择动作,返回执行动作\n",
    "    Epsilon=Epsilon_stop+(Epsilon_start-Epsilon_stop)*np.exp(-decay_rate*ep)\n",
    "    States_Actions=Q_Table.loc[[State],:]#选择目前状态下，取出Q表的目前的状态行,state选择为数字\n",
    "    if(np.random.rand()<Epsilon or np.all(States_Actions==[0])):#其实np.all(States_Action==[0])也行\n",
    "        Execute_Action=np.random.choice(Actions)\n",
    "    else:\n",
    "        #Q_T=Q_Table.T\n",
    "        #G=(Q_Table.T).loc[:,[State]]\n",
    "        Execute_Action=np.array(pd.DataFrame.idxmax((Q_Table.T).loc[:,[State]]))[0]#这里真心累，有毒\n",
    "    return Execute_Action\n",
    "#———————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Next_State(State,Execute_Action):#根据目前状态个选择的动作,返回即时奖励和新状态\n",
    "    if(Execute_Action=='left'):\n",
    "        #目标在右上角，如果Execute_Action为‘left’，不可能到达目标\n",
    "        if(State%N_Mesh==1):#判断是不是在网格最左边\n",
    "            State=State\n",
    "            R=-100\n",
    "        elif((State+1) in Obstable):\n",
    "            State=State-1\n",
    "            R=-100\n",
    "        else:\n",
    "            State=State-1\n",
    "            R=0\n",
    "    elif(Execute_Action=='right'):\n",
    "        if(State==N_States-1):\n",
    "            State='End'\n",
    "            R=100\n",
    "        elif(State%N_Mesh==0):\n",
    "            State=State\n",
    "            R=-100\n",
    "        elif((State+1) in Obstable):\n",
    "            State=State+1\n",
    "            R=-100\n",
    "        else:\n",
    "            State=State+1\n",
    "            R=0.2\n",
    "    elif(Execute_Action=='up'):\n",
    "        if(State==N_States-N_Mesh):\n",
    "            State='End'\n",
    "            R=100\n",
    "        elif(N_States+1-N_Mesh<=State<=N_States-1):\n",
    "            State=State\n",
    "            R=-100\n",
    "        elif((State+10) in Obstable):\n",
    "            State=State+10\n",
    "            R=-100\n",
    "        else:\n",
    "            State=State+N_Mesh\n",
    "            R=0.2\n",
    "    else:\n",
    "        if(0<=State<=N_Mesh):\n",
    "            State=State\n",
    "            R=-100\n",
    "        elif((State-10) in Obstable):\n",
    "            State=State-10\n",
    "            R=-0.2\n",
    "        else:\n",
    "            R=0\n",
    "            State=State-10\n",
    "    return State,R\n",
    "        \n",
    "#————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Run_Function():\n",
    "    #——————————————————————————————————————————————————画图程序部分\n",
    "    \n",
    "    \n",
    "    #——————————————————————————————————————————————————\n",
    "    Q_Table=Initial_Q_Table (N_States,Actions)\n",
    "    Step=0 \n",
    "    for episode in np.arange(300) :#跑20回合\n",
    "        if(episode%300==0):\n",
    "            print('Q_Table,%d'%episode)\n",
    "            Q_Table\n",
    "#         fig=plt.figure()\n",
    "#         ax=fig.gca()\n",
    "#         ax.set(xlim=[0, N_Mesh], ylim=[0, N_Mesh])\n",
    "#         ax.set_xticks(np.arange(0,(N_Mesh+1)))\n",
    "#         ax.set_yticks(np.arange(0,(N_Mesh+1)))\n",
    "#         plt.grid()\n",
    "        print('episode is %d' % episode)\n",
    "        if(np.random.rand()<0.4):\n",
    "            State_=1 #定义起点 \n",
    "            print(\"Normal_start:%d\"%State_)\n",
    "        else:\n",
    "            State_=np.random.randint(low=5, high=100)\n",
    "            print(\"Random_start:%d\"%State_)\n",
    "        Is_End=False\n",
    "        while not Is_End:\n",
    "            A=Select_Action(State_,Q_Table,Step)\n",
    "            Next_S,R=Next_State(State_,A)\n",
    "            #print(Next_S)\n",
    "            Q_Table_S_A=Q_Table.loc[[State_],[A]]\n",
    "            if Next_S!='End':\n",
    "                D_T=np.array((Q_Table.loc[[Next_S],:]).T)\n",
    "                Q_Target=R+Beta*D_T.max() #Next_S状态中的可选动作中的最大Q值动作\n",
    "                Step+=1\n",
    "            else:\n",
    "                Q_Target=R\n",
    "                Step+=1\n",
    "                Is_End=True #目的是结束循\n",
    "#————————————————————————————————————————————————————————————————————————————画图程序\n",
    "#             X_State,Y_State=Caculate_coordinate(State_)\n",
    "#             if(Next_S)=='End':\n",
    "#                 H=N_States\n",
    "#             else:\n",
    "#                 H=Next_S\n",
    "#             X_Next_State,Y_Next_State=Caculate_coordinate(H)                      \n",
    "#             if(Step==0):\n",
    "#                 plt.scatter(X_State,Y_State)  \n",
    "#             else:\n",
    "#                 plt.scatter(X_State,Y_State) \n",
    "#                 plt.scatter(X_Next_State,Y_Next_State)\n",
    "#                 plt.plot([X_State,X_Next_State],[Y_State,Y_Next_State])\n",
    "#————————————————————————————————————————————————————————————————————————————\n",
    "            Q_Table.loc[[State_],[A]]+=Alpha*(Q_Target-Q_Table_S_A)\n",
    "            State_=Next_S\n",
    "        print(Step)\n",
    "        print(Epsilon_stop+(Epsilon_start-Epsilon_stop)*np.exp(-decay_rate*Step))\n",
    "        #plt.show()\n",
    "        #print(Q_Table)\n",
    "    return Q_Table\n",
    "#————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "Q__Table=Run_Function()\n",
    "Q__Table\n",
    "Final_Q_Table=np.array(Q__Table)\n",
    "#—————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Plot_Final_Graph(Q_TABLE):\n",
    "    fig=plt.figure()\n",
    "    ax=fig.gca()\n",
    "    ax.set(xlim=[0, N_Mesh], ylim=[0, N_Mesh])\n",
    "    ax.set_xticks(np.arange(0,(N_Mesh+1)))\n",
    "    ax.set_yticks(np.arange(0,(N_Mesh+1)))\n",
    "    plt.grid()\n",
    "    State_Start=1\n",
    "    State=State_Start\n",
    "    Step=1\n",
    "    while(State!=N_States):\n",
    "        Action=np.argmax((Q_TABLE[State-1]))\n",
    "        if(Action==0):\n",
    "             Next_State=State-1\n",
    "        elif(Action==1):\n",
    "             Next_State=State+1\n",
    "        elif(Action==2):\n",
    "             Next_State=State+N_Mesh\n",
    "        else:\n",
    "             Next_State=State-N_Mesh\n",
    "        Step+=1\n",
    "        print(Next_State)\n",
    "        X_State,Y_State=Caculate_coordinate(State)\n",
    "        X_Next_State,Y_Next_State=Caculate_coordinate(Next_State)\n",
    "        if(Step==0):\n",
    "            plt.scatter(X_State,Y_State)  \n",
    "        else:\n",
    "            plt.scatter(X_State,Y_State) \n",
    "            plt.scatter(X_Next_State,Y_Next_State)\n",
    "            plt.plot([X_State,X_Next_State],[Y_State,Y_Next_State])\n",
    "        for J in np.arange(len(Obstable)):\n",
    "            plt.scatter((Caculate_coordinate(Obstable[J]))[0],(Caculate_coordinate(Obstable[J]))[1],marker=\"x\")\n",
    "        print(X_State,Y_State)\n",
    "        print(X_Next_State,Y_Next_State)\n",
    "        State=Next_State\n",
    "    plt.show()\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "Plot_Final_Graph(Final_Q_Table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
